package com.youth.fund.hk.hostManager.lucene.webcontent;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.youth.fund.hk.hostManager.lucene.datamodel.ContentModel;

public class WebContent 
{
	/**
	 * 读取一个网页全部内容
	 */
	public String getOneHtml(final String htmlurl) throws IOException {
		URL url;
		String temp;
		final StringBuffer sb = new StringBuffer();
		try {
			url = new URL(htmlurl);
			final BufferedReader in = new BufferedReader(new InputStreamReader(
					url.openStream(), "utf-8"));// 读取网页全部内容
			while ((temp = in.readLine()) != null) {
				sb.append(temp);
			}
			in.close();
		} catch (final MalformedURLException me) {
			me.getMessage();
			throw me;
		} catch (final IOException e) {
			e.printStackTrace();
			throw e;
		}
		return sb.toString();
	}


	/**
	 *  Function:抓取网页内容
	 *  @author JLC
	 *  @param url
	 *  @return
	 */
	public ContentModel getContentFromSite(String url){
		ContentModel cobj= new ContentModel();
		String html = "";
		try {
			html = getOneHtml(url);
			
			Document doc = Jsoup.connect(url).get();
			
			cobj.setTitle(doc.title());
			
			cobj.setContent(doc.getElementById("main_container").text());
			
			cobj.setUrl(url);
		} catch (final Exception e) {}
		return cobj;
	}
}
